# -*- coding: utf-8 -*-
import re
import datetime


import scrapy
from scrapy.http import Request
from urllib import parse
from  ArticleSpider.items import JobBoleArticleItem,ArticleItemLoader
from ArticleSpider.util.common import get_md5
from scrapy.loader import ItemLoader
from scrapy.xlib.pydispatch import dispatcher
from scrapy import signals

from selenium import webdriver

from pyvirtualdisplay import Display

class JobboleSpider(scrapy.Spider):
    name = 'jobbole'
    allowed_domains = ['blog.jobbole.com']
    start_urls = ['http://blog.jobbole.com/all-posts/']

    # 404url
    # start_urls = ['http://blog.jobbole.com/ixuea/']
    # start_urls = ['https://www.ip.cn/']

    # def __init__(self):
    #     super(JobboleSpider, self).__init__()
    #
    #     # 创建一个显示器
    #     # visible=0:表示不显示窗口
    #     # 后面的表示窗口分辨率（显示器分辨率）
    #     display=Display(visible=0,size=(800,600))
    #
    #     # 开启
    #     display.start()
    #
    #     self.browser = webdriver.Chrome()
    #
    #     # 信号量(可以简单理解为生命周期方法)关联到一个方法
    #     # 这里是将spider关闭时回调关联为pider_closed方法
    #     dispatcher.connect(self.spider_closed,signals.spider_closed)
    #
    # def spider_closed(self,spider):
    #     print("jobbole spider closed")
    #     self.browser.quit()

    # 收集所有404页面，url，数量
    # handle_httpstatus_list=[404]

    def __init__(self,**kwargs):
        self.fail_urls=[]

    def parse(self, response):
        '''
        获取当前页面，所有文章url
        获取当前页面下一节urldatetime
        :param response:
        :return:
        '''

        # if response.status==404:
        #     self.fail_urls.append(response.url)
        #     self.crawler.stats.inc_value("fail_url_counts")

        #获取当前页面，所有文章url
        post_nodes=response.css("#archive .floated-thumb .post-thumb a")
        for post_node in post_nodes:
            #如果提取到的地址，没有域名
            #parse.urljoin(response.url,post_url)
            cover_image=post_node.css("img::attr(src)").extract_first()
            post_url=post_node.css("::attr(href)").extract_first()
            yield Request(url=post_url,meta={"cover_image":cover_image},callback=self.parse_detail)

        #提取下一页
        # next_url=response.css(".next.page-numbers::attr(href)").extract_first("")
        # if next_url:
        #     yield Request(url=next_url, callback=self.parse)

    def make_requests_from_url(self, url):
        """ This method is deprecated. """
        # return Request(url, dont_filter=True,meta={"proxy":"http://49.86.177.193:9999"})
        return Request(url, dont_filter=True)

    def parse_detail(self, response):
        # title=response.xpath('//div[@class="entry-header"]/h1/text()').extract()[0]
        # created_at=response.xpath("//p[@class='entry-meta-hide-on-mobile']/text()").extract()[0].strip().replace("·","").strip()
        # praise_nums=int(response.xpath("//span[contains(@class,'vote-post-up')]/h10/text()").extract()[0])
        #
        # fav_nums=response.xpath("//span[contains(@class,'bookmark-btn')]/text()").extract()[0]
        # fav_nums_re=re.match(".*?(\d+).*",fav_nums)
        # if fav_nums_re:
        #     #从1开始
        #     fav_nums=fav_nums_re.group(1)
        #
        # comment_nums=response.xpath("//a[@href='#article-comment']/text()").extract()[0]
        # comment_nums_re = re.match(".*?(\d+).*", comment_nums)
        # if comment_nums_re:
        #     comment_nums = comment_nums_re.group(1)
        #
        # cotent=response.xpath("//div[@class='entry']").extract()[0]
        #
        # tags=response.xpath('//p[@class="entry-meta-hide-on-mobile"]/a/text()').extract()
        #
        # #去除评论
        # tags=[e for e in tags if not e.strip().endswith("评论")]
        #
        # tagString=",".join(tags)

        #文章封面图
        # cover_image=response.meta.get("cover_image","")

        #css选择器

        # title=response.css(".entry-header h1::text").extract_first()
        # created_at =response.css("p.entry-meta-hide-on-mobile::text").extract()[0].strip().replace("·","").strip()
        # praise_nums = int(response.css(".vote-post-up h10::text").extract()[0])
        #
        # fav_nums = response.css("span.bookmark-btn::text").extract()[0]
        #
        # #去掉左侧，贪婪模式
        # fav_nums_re=re.match(".*?(\d+).*",fav_nums)
        # if fav_nums_re:
        #     #从1开始
        #     fav_nums=int(fav_nums_re.group(1))
        # else:
        #     fav_nums=0
        #
        # comment_nums=response.css("a[href='#article-comment'] span::text").extract()[0]
        # comment_nums_re = re.match(".*?(\d+).*", comment_nums)
        # if comment_nums_re:
        #     comment_nums = int(comment_nums_re.group(1))
        # else:
        #     comment_nums=0
        #
        # content = response.css("div.entry").extract()[0]
        #
        # tags=response.css(".entry-meta-hide-on-mobile a::text").extract()
        #
        # #去除评论
        # tags=[e for e in tags if not e.strip().endswith("评论")]
        #
        # tagString=",".join(tags)
        #
        # # print(title)
        #
        # articleItem=JobBoleArticleItem()
        # articleItem["title"]=title
        #
        # #将日期字符串，转为datetime
        # try:
        #     created_at=datetime.datetime.struct_time(created_at,"%Y/%m/%d").date()
        # except Exception as e:
        #     #如果出错了，当前时间
        #     created_at=datetime.datetime.now().date()
        #
        # articleItem["created_at"]=created_at
        # articleItem["url"]=response.url
        # articleItem["cover_image"]=[cover_image]
        # articleItem["praise_nums"]=praise_nums
        # articleItem["comment_nums"]=comment_nums
        # articleItem["fav_nums"]=fav_nums
        # articleItem["tags"]=tags
        # articleItem["content"]=content
        # articleItem["url_object_id"]=get_md5(response.url)

        #通过ItemLoader加载item
        item_loader=ArticleItemLoader(item=JobBoleArticleItem(),response=response)

        #表示item字段的值，通过response的css选择器获取
        item_loader.add_css("title",".entry-header h1::text")

        #添加一个直接的值
        cover_image = response.meta.get("cover_image", "")

        item_loader.add_value("url",response.url)
        item_loader.add_value("url_object_id",get_md5(response.url))
        item_loader.add_css("created_at", "p.entry-meta-hide-on-mobile::text")
        item_loader.add_value("cover_image", cover_image)
        item_loader.add_css("praise_nums", ".vote-post-up h10::text")
        item_loader.add_css("comment_nums", "a[href='#article-comment'] span::text")
        item_loader.add_css("fav_nums", "span.bookmark-btn::text")
        item_loader.add_css("tags", ".entry-meta-hide-on-mobile a::text")
        item_loader.add_css("content", "div.entry")

        articleItem=item_loader.load_item()
        yield articleItem

